-      }.should_not change { @sent_posts.length }
105 
+        }.should change { @sent_requests[Net::HTTP::Get].length }.by(1)
106 
+      }.should_not change { @sent_requests[Net::HTTP::Post].length }
84 107
85 
-      @sent_gets[0].should == @checker.options['payload']
108 
+      @sent_requests[Net::HTTP::Get][0].should == @checker.options['payload']
86 109 
     end
87 110 
   end
88 111 
 
@@ -112,7 +135,7 @@ describe Agents::PostAgent do
112 135 
       @checker.should_not be_valid
113 136 
     end
114 137
115 
-    it "should validate method as post or get, defaulting to post" do
138 
+    it "should validate method as post, get, put, patch, or delete, defaulting to post" do
116 139 
       @checker.options['method'] = ""
117 140 
       @checker.method.should == "post"
118 141 
       @checker.should be_valid
@@ -125,11 +148,35 @@ describe Agents::PostAgent do
125 148 
       @checker.method.should == "get"
126 149 
       @checker.should be_valid
127 150
151 
+      @checker.options['method'] = "patch"
152 
+      @checker.method.should == "patch"
153 
+      @checker.should be_valid
154 
+
128 155 
       @checker.options['method'] = "wut"
129 156 
       @checker.method.should == "wut"
130 157 
       @checker.should_not be_valid
131 158 
     end
132 159
160 
+    it "should validate that no_merge is 'true' or 'false', if present" do
161 
+      @checker.options['no_merge'] = ""
162 
+      @checker.should be_valid
163 
+
164 
+      @checker.options['no_merge'] = "true"
165 
+      @checker.should be_valid
166 
+
167 
+      @checker.options['no_merge'] = "false"
168 
+      @checker.should be_valid
169 
+
170 
+      @checker.options['no_merge'] = false
171 
+      @checker.should be_valid
172 
+
173 
+      @checker.options['no_merge'] = true
174 
+      @checker.should be_valid
175 
+
176 
+      @checker.options['no_merge'] = 'blarg'
177 
+      @checker.should_not be_valid
178 
+    end
179 
+
133 180 
     it "should validate payload as a hash, if present" do
134 181 
       @checker.options['payload'] = ""
135 182 
       @checker.should be_valid
@@ -178,7 +225,17 @@ describe Agents::PostAgent do
178 225 
     it "just returns the post_uri when no params are given" do
179 226 
       @checker.options['post_url'] = "http://example.com/a/path?existing_param=existing_value"
180 227 
       uri = @checker.generate_uri
228 
+      uri.host.should == 'example.com'
229 
+      uri.scheme.should == 'http'
181 230 
       uri.request_uri.should == "/a/path?existing_param=existing_value"
182 231 
     end
232 
+
233 
+    it "interpolates when receiving a payload" do
234 
+      @checker.options['post_url'] = "https://{{ domain }}/{{ variable }}?existing_param=existing_value"
235 
+      uri = @checker.generate_uri({ "some_param" => "some_value", "another_param" => "another_value" }, { 'domain' => 'google.com', 'variable' => 'a_variable' })
236 
+      uri.request_uri.should == "/a_variable?existing_param=existing_value&some_param=some_value&another_param=another_value"
237 
+      uri.host.should == 'google.com'
238 
+      uri.scheme.should == 'https'
239 
+    end
183 240 
   end
184 241 
 end

+ 81 - 0
spec/models/agents/rss_agent_spec.rb
Просмотреть файл 

@@ -0,0 +1,81 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::RssAgent do
4 
+  before do
5 
+    @valid_options = {
6 
+      'expected_update_period_in_days' => "2",
7 
+      'url' => "https://github.com/cantino/huginn/commits/master.atom",
8 
+    }
9 
+
10 
+    stub_request(:any, /github.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/github_rss.atom")), :status => 200)
11 
+  end
12 
+
13 
+  let(:agent) do
14 
+    _agent = Agents::RssAgent.new(:name => "github rss feed", :options => @valid_options)
15 
+    _agent.user = users(:bob)
16 
+    _agent.save!
17 
+    _agent
18 
+  end
19 
+
20 
+  it_behaves_like WebRequestConcern
21 
+
22 
+  describe "validations" do
23 
+    it "should validate the presence of url" do
24 
+      agent.options['url'] = "http://google.com"
25 
+      agent.should be_valid
26 
+
27 
+      agent.options['url'] = ""
28 
+      agent.should_not be_valid
29 
+
30 
+      agent.options['url'] = nil
31 
+      agent.should_not be_valid
32 
+    end
33 
+
34 
+    it "should validate the presence and numericality of expected_update_period_in_days" do
35 
+      agent.options['expected_update_period_in_days'] = "5"
36 
+      agent.should be_valid
37 
+
38 
+      agent.options['expected_update_period_in_days'] = "wut?"
39 
+      agent.should_not be_valid
40 
+
41 
+      agent.options['expected_update_period_in_days'] = 0
42 
+      agent.should_not be_valid
43 
+
44 
+      agent.options['expected_update_period_in_days'] = nil
45 
+      agent.should_not be_valid
46 
+
47 
+      agent.options['expected_update_period_in_days'] = ""
48 
+      agent.should_not be_valid
49 
+    end
50 
+  end
51 
+
52 
+  describe "emitting RSS events" do
53 
+    it "should emit items as events" do
54 
+      lambda {
55 
+        agent.check
56 
+      }.should change { agent.events.count }.by(20)
57 
+    end
58 
+
59 
+    it "should track ids and not re-emit the same item when seen again" do
60 
+      agent.check
61 
+      agent.memory['seen_ids'].should == agent.events.map {|e| e.payload['id'] }
62 
+
63 
+      newest_id = agent.memory['seen_ids'][0]
64 
+      agent.events.first.payload['id'].should == newest_id
65 
+      agent.memory['seen_ids'] = agent.memory['seen_ids'][1..-1] # forget the newest id
66 
+
67 
+      lambda {
68 
+        agent.check
69 
+      }.should change { agent.events.count }.by(1)
70 
+
71 
+      agent.events.first.payload['id'].should == newest_id
72 
+      agent.memory['seen_ids'][0].should == newest_id
73 
+    end
74 
+
75 
+    it "should truncate the seen_ids in memory at 500 items" do
76 
+      agent.memory['seen_ids'] = ['x'] * 490
77 
+      agent.check
78 
+      agent.memory['seen_ids'].length.should == 500
79 
+    end
80 
+  end
81 
+end

+ 85 - 60
spec/models/agents/website_agent_spec.rb
Просмотреть файл 

@@ -4,23 +4,25 @@ describe Agents::WebsiteAgent do
4 4 
   describe "checking without basic auth" do
5 5 
     before do
6 6 
       stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
7 
-      @site = {
7 
+      @valid_options = {
8 8 
         'name' => "XKCD",
9 
-        'expected_update_period_in_days' => 2,
9 
+        'expected_update_period_in_days' => "2",
10 10 
         'type' => "html",
11 11 
         'url' => "http://xkcd.com",
12 12 
         'mode' => 'on_change',
13 13 
         'extract' => {
14 
-          'url' => { 'css' => "#comic img", 'attr' => "src" },
15 
-          'title' => { 'css' => "#comic img", 'attr' => "alt" },
16 
-          'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
14 
+          'url' => { 'css' => "#comic img", 'value' => "@src" },
15 
+          'title' => { 'css' => "#comic img", 'value' => "@alt" },
16 
+          'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
17 17 
         }
18 18 
       }
19 
-      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @site, :keep_events_for => 2)
19 
+      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2)
20 20 
       @checker.user = users(:bob)
21 21 
       @checker.save!
22 22 
     end
23 23
24 
+    it_behaves_like WebRequestConcern
25 
+
24 26 
     describe "validations" do
25 27 
       before do
26 28 
         @checker.should be_valid
 
@@ -42,20 +44,6 @@ describe Agents::WebsiteAgent do
42 44 
         @checker.should be_valid
43 45 
       end
44 46
45 
-      it "should validate headers" do
46 
-        @checker.options['headers'] = "blah"
47 
-        @checker.should_not be_valid
48 
-
49 
-        @checker.options['headers'] = ""
50 
-        @checker.should be_valid
51 
-
52 
-        @checker.options['headers'] = {}
53 
-        @checker.should be_valid
54 
-
55 
-        @checker.options['headers'] = { 'foo' => 'bar' }
56 
-        @checker.should be_valid
57 
-      end
58 
-
59 47 
       it "should validate mode" do
60 48 
         @checker.options['mode'] = "nonsense"
61 49 
         @checker.should_not be_valid
 
@@ -97,16 +85,16 @@ describe Agents::WebsiteAgent do
97 85
98 86 
       it "should always save events when in :all mode" do
99 87 
         lambda {
100 
-          @site['mode'] = 'all'
101 
-          @checker.options = @site
88 
+          @valid_options['mode'] = 'all'
89 
+          @checker.options = @valid_options
102 90 
           @checker.check
103 91 
           @checker.check
104 92 
         }.should change { Event.count }.by(2)
105 93 
       end
106 94
107 95 
       it "should take uniqueness_look_back into account during deduplication" do
108 
-        @site['mode'] = 'all'
109 
-        @checker.options = @site
96 
+        @valid_options['mode'] = 'all'
97 
+        @checker.options = @valid_options
110 98 
         @checker.check
111 99 
         @checker.check
112 100 
         event = Event.last
 
@@ -114,47 +102,47 @@ describe Agents::WebsiteAgent do
114 102 
         event.save
115 103
116 104 
         lambda {
117 
-          @site['mode'] = 'on_change'
118 
-          @site['uniqueness_look_back'] = 2
119 
-          @checker.options = @site
105 
+          @valid_options['mode'] = 'on_change'
106 
+          @valid_options['uniqueness_look_back'] = 2
107 
+          @checker.options = @valid_options
120 108 
           @checker.check
121 109 
         }.should_not change { Event.count }
122 110
123 111 
         lambda {
124 
-          @site['mode'] = 'on_change'
125 
-          @site['uniqueness_look_back'] = 1
126 
-          @checker.options = @site
112 
+          @valid_options['mode'] = 'on_change'
113 
+          @valid_options['uniqueness_look_back'] = 1
114 
+          @checker.options = @valid_options
127 115 
           @checker.check
128 116 
         }.should change { Event.count }.by(1)
129 117 
       end
130 118
131 119 
       it "should log an error if the number of results for a set of extraction patterns differs" do
132 
-        @site['extract']['url']['css'] = "div"
133 
-        @checker.options = @site
120 
+        @valid_options['extract']['url']['css'] = "div"
121 
+        @checker.options = @valid_options
134 122 
         @checker.check
135 123 
         @checker.logs.first.message.should =~ /Got an uneven number of matches/
136 124 
       end
137 125
138 126 
       it "should accept an array for url" do
139 
-        @site['url'] = ["http://xkcd.com/1/", "http://xkcd.com/2/"]
140 
-        @checker.options = @site
127 
+        @valid_options['url'] = ["http://xkcd.com/1/", "http://xkcd.com/2/"]
128 
+        @checker.options = @valid_options
141 129 
         lambda { @checker.save! }.should_not raise_error;
142 130 
         lambda { @checker.check }.should_not raise_error;
143 131 
       end
144 132
145 133 
       it "should parse events from all urls in array" do
146 134 
         lambda {
147 
-          @site['url'] = ["http://xkcd.com/", "http://xkcd.com/"]
148 
-          @site['mode'] = 'all'
149 
-          @checker.options = @site
135 
+          @valid_options['url'] = ["http://xkcd.com/", "http://xkcd.com/"]
136 
+          @valid_options['mode'] = 'all'
137 
+          @checker.options = @valid_options
150 138 
           @checker.check
151 139 
         }.should change { Event.count }.by(2)
152 140 
       end
153 141
154 142 
       it "should follow unique rules when parsing array of urls" do
155 143 
         lambda {
156 
-          @site['url'] = ["http://xkcd.com/", "http://xkcd.com/"]
157 
-          @checker.options = @site
144 
+          @valid_options['url'] = ["http://xkcd.com/", "http://xkcd.com/"]
145 
+          @checker.options = @valid_options
158 146 
           @checker.check
159 147 
         }.should change { Event.count }.by(1)
160 148 
       end
 
@@ -170,7 +158,7 @@ describe Agents::WebsiteAgent do
170 158 
           }, :status => 200)
171 159 
         site = {
172 160 
           'name' => "Some JSON Response",
173 
-          'expected_update_period_in_days' => 2,
161 
+          'expected_update_period_in_days' => "2",
174 162 
           'type' => "json",
175 163 
           'url' => "http://no-encoding.example.com",
176 164 
           'mode' => 'on_change',
 
@@ -197,7 +185,7 @@ describe Agents::WebsiteAgent do
197 185 
           }, :status => 200)
198 186 
         site = {
199 187 
           'name' => "Some JSON Response",
200 
-          'expected_update_period_in_days' => 2,
188 
+          'expected_update_period_in_days' => "2",
201 189 
           'type' => "json",
202 190 
           'url' => "http://wrong-encoding.example.com",
203 191 
           'mode' => 'on_change',
 
@@ -248,11 +236,11 @@ describe Agents::WebsiteAgent do
248 236 
       end
249 237
250 238 
       it "parses XPath" do
251 
-        @site['extract'].each { |key, value|
239 
+        @valid_options['extract'].each { |key, value|
252 240 
           value.delete('css')
253 241 
           value['xpath'] = "//*[@id='comic']//img"
254 242 
         }
255 
-        @checker.options = @site
243 
+        @checker.options = @valid_options
256 244 
         @checker.check
257 245 
         event = Event.last
258 246 
         event.payload['url'].should == "http://imgs.xkcd.com/comics/evolving.png"
 
@@ -263,13 +251,12 @@ describe Agents::WebsiteAgent do
263 251 
       it "should turn relative urls to absolute" do
264 252 
         rel_site = {
265 253 
           'name' => "XKCD",
266 
-          'expected_update_period_in_days' => 2,
254 
+          'expected_update_period_in_days' => "2",
267 255 
           'type' => "html",
268 256 
           'url' => "http://xkcd.com",
269 257 
           'mode' => "on_change",
270 258 
           'extract' => {
271 
-            'url' => {'css' => "#topLeft a", 'attr' => "href"},
272 
-            'title' => {'css' => "#topLeft a", 'text' => "true"}
259 
+            'url' => {'css' => "#topLeft a", 'value' => "@href"},
273 260 
           }
274 261 
         }
275 262 
         rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
 
@@ -280,6 +267,44 @@ describe Agents::WebsiteAgent do
280 267 
         event.payload['url'].should == "http://xkcd.com/about"
281 268 
       end
282 269
270 
+      it "should return an integer value if XPath evaluates to one" do
271 
+        rel_site = {
272 
+          'name' => "XKCD",
273 
+          'expected_update_period_in_days' => 2,
274 
+          'type' => "html",
275 
+          'url' => "http://xkcd.com",
276 
+          'mode' => "on_change",
277 
+          'extract' => {
278 
+            'num_links' => {'css' => "#comicLinks", 'value' => "count(./a)"}
279 
+          }
280 
+        }
281 
+        rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
282 
+        rel.user = users(:bob)
283 
+        rel.save!
284 
+        rel.check
285 
+        event = Event.last
286 
+        event.payload['num_links'].should == "9"
287 
+      end
288 
+
289 
+      it "should return all texts concatenated if XPath returns many text nodes" do
290 
+        rel_site = {
291 
+          'name' => "XKCD",
292 
+          'expected_update_period_in_days' => 2,
293 
+          'type' => "html",
294 
+          'url' => "http://xkcd.com",
295 
+          'mode' => "on_change",
296 
+          'extract' => {
297 
+            'slogan' => {'css' => "#slogan", 'value' => ".//text()"}
298 
+          }
299 
+        }
300 
+        rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
301 
+        rel.user = users(:bob)
302 
+        rel.save!
303 
+        rel.check
304 
+        event = Event.last
305 
+        event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language."
306 
+      end
307 
+
283 308 
       describe "JSON" do
284 309 
         it "works with paths" do
285 310 
           json = {
 
@@ -291,7 +316,7 @@ describe Agents::WebsiteAgent do
291 316 
           stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
292 317 
           site = {
293 318 
             'name' => "Some JSON Response",
294 
-            'expected_update_period_in_days' => 2,
319 
+            'expected_update_period_in_days' => "2",
295 320 
             'type' => "json",
296 321 
             'url' => "http://json-site.com",
297 322 
             'mode' => 'on_change',
 
@@ -322,7 +347,7 @@ describe Agents::WebsiteAgent do
322 347 
           stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
323 348 
           site = {
324 349 
             'name' => "Some JSON Response",
325 
-            'expected_update_period_in_days' => 2,
350 
+            'expected_update_period_in_days' => "2",
326 351 
             'type' => "json",
327 352 
             'url' => "http://json-site.com",
328 353 
             'mode' => 'on_change',
 
@@ -358,7 +383,7 @@ describe Agents::WebsiteAgent do
358 383 
           stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
359 384 
           site = {
360 385 
             'name' => "Some JSON Response",
361 
-            'expected_update_period_in_days' => 2,
386 
+            'expected_update_period_in_days' => "2",
362 387 
             'type' => "json",
363 388 
             'url' => "http://json-site.com",
364 389 
             'mode' => 'on_change'
 
@@ -382,7 +407,7 @@ describe Agents::WebsiteAgent do
382 407 
         @event.payload = { 'url' => "http://xkcd.com" }
383 408
384 409 
         lambda {
385 
-          @checker.options = @site
410 
+          @checker.options = @valid_options
386 411 
           @checker.receive([@event])
387 412 
         }.should change { Event.count }.by(1)
388 413 
       end
 
@@ -394,20 +419,20 @@ describe Agents::WebsiteAgent do
394 419 
       stub_request(:any, /example/).
395 420 
         with(headers: { 'Authorization' => "Basic #{['user:pass'].pack('m').chomp}" }).
396 421 
         to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
397 
-      @site = {
422 
+      @valid_options = {
398 423 
         'name' => "XKCD",
399 
-        'expected_update_period_in_days' => 2,
424 
+        'expected_update_period_in_days' => "2",
400 425 
         'type' => "html",
401 426 
         'url' => "http://www.example.com",
402 427 
         'mode' => 'on_change',
403 428 
         'extract' => {
404 
-          'url' => { 'css' => "#comic img", 'attr' => "src" },
405 
-          'title' => { 'css' => "#comic img", 'attr' => "alt" },
406 
-          'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
429 
+          'url' => { 'css' => "#comic img", 'value' => "@src" },
430 
+          'title' => { 'css' => "#comic img", 'value' => "@alt" },
431 
+          'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
407 432 
         },
408 433 
         'basic_auth' => "user:pass"
409 434 
       }
410 
-      @checker = Agents::WebsiteAgent.new(:name => "auth", :options => @site)
435 
+      @checker = Agents::WebsiteAgent.new(:name => "auth", :options => @valid_options)
411 436 
       @checker.user = users(:bob)
412 437 
       @checker.save!
413 438 
     end
 
@@ -425,18 +450,18 @@ describe Agents::WebsiteAgent do
425 450 
       stub_request(:any, /example/).
426 451 
         with(headers: { 'foo' => 'bar', 'user_agent' => /Faraday/ }).
427 452 
         to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
428 
-      @site = {
453 
+      @valid_options = {
429 454 
         'name' => "XKCD",
430 
-        'expected_update_period_in_days' => 2,
455 
+        'expected_update_period_in_days' => "2",
431 456 
         'type' => "html",
432 457 
         'url' => "http://www.example.com",
433 458 
         'mode' => 'on_change',
434 459 
         'headers' => { 'foo' => 'bar' },
435 460 
         'extract' => {
436 
-          'url' => { 'css' => "#comic img", 'attr' => "src" },
461 
+          'url' => { 'css' => "#comic img", 'value' => "@src" },
437 462 
         }
438 463 
       }
439 
-      @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @site)
464 
+      @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options)
440 465 
       @checker.user = users(:bob)
441 466 
       @checker.save!
442 467 
     end

+ 36 - 0
spec/models/event_spec.rb
Просмотреть файл 

@@ -76,3 +76,39 @@ describe Event do
76 76 
     end
77 77 
   end
78 78 
 end
79 
+
80 
+describe EventDrop do
81 
+  def interpolate(string, event)
82 
+    event.agent.interpolate_string(string, event.to_liquid)
83 
+  end
84 
+
85 
+  before do
86 
+    @event = Event.new
87 
+    @event.agent = agents(:jane_weather_agent)
88 
+    @event.payload = {
89 
+      'title' => 'some title',
90 
+      'url' => 'http://some.site.example.org/',
91 
+    }
92 
+    @event.save!
93 
+  end
94 
+
95 
+  it 'should be created via Agent#to_liquid' do
96 
+    @event.to_liquid.class.should be(EventDrop)
97 
+  end
98 
+
99 
+  it 'should have attributes of its payload' do
100 
+    t = '{{title}}: {{url}}'
101 
+    interpolate(t, @event).should eq('some title: http://some.site.example.org/')
102 
+  end
103 
+
104 
+  it 'should be iteratable' do
105 
+    # to_liquid returns self
106 
+    t = "{% for pair in to_liquid %}{{pair | join:':' }}\n{% endfor %}"
107 
+    interpolate(t, @event).should eq("title:some title\nurl:http://some.site.example.org/\n")
108 
+  end
109 
+
110 
+  it 'should have agent' do
111 
+    t = '{{agent.name}}'
112 
+    interpolate(t, @event).should eq('SF Weather')
113 
+  end
114 
+end

+ 88 - 0
spec/support/shared_examples/email_concern.rb
Просмотреть файл 

@@ -0,0 +1,88 @@
1 
+require 'spec_helper'
2 
+
3 
+shared_examples_for EmailConcern do
4 
+  let(:valid_options) {
5 
+    {
6 
+      :subject => "hello!",
7 
+      :expected_receive_period_in_days => "2"
8 
+    }
9 
+  }
10 
+
11 
+  let(:agent) do
12 
+    _agent = described_class.new(:name => "some email agent", :options => valid_options)
13 
+    _agent.user = users(:jane)
14 
+    _agent
15 
+  end
16 
+
17 
+  describe "validations" do
18 
+    it "should be valid" do
19 
+      agent.should be_valid
20 
+    end
21 
+
22 
+    it "should validate the presence of 'subject'" do
23 
+      agent.options['subject'] = ''
24 
+      agent.should_not be_valid
25 
+
26 
+      agent.options['subject'] = nil
27 
+      agent.should_not be_valid
28 
+    end
29 
+
30 
+    it "should validate the presence of 'expected_receive_period_in_days'" do
31 
+      agent.options['expected_receive_period_in_days'] = ''
32 
+      agent.should_not be_valid
33 
+
34 
+      agent.options['expected_receive_period_in_days'] = nil
35 
+      agent.should_not be_valid
36 
+    end
37 
+
38 
+    it "should validate that recipients, when provided, is one or more valid email addresses" do
39 
+      agent.options['recipients'] = ''
40 
+      agent.should be_valid
41 
+
42 
+      agent.options['recipients'] = nil
43 
+      agent.should be_valid
44 
+
45 
+      agent.options['recipients'] = 'bob@example.com'
46 
+      agent.should be_valid
47 
+
48 
+      agent.options['recipients'] = ['bob@example.com']
49 
+      agent.should be_valid
50 
+
51 
+      agent.options['recipients'] = ['bob@example.com', 'jane@example.com']
52 
+      agent.should be_valid
53 
+
54 
+      agent.options['recipients'] = ['bob@example.com', 'example.com']
55 
+      agent.should_not be_valid
56 
+
57 
+      agent.options['recipients'] = ['hi!']
58 
+      agent.should_not be_valid
59 
+
60 
+      agent.options['recipients'] = { :foo => "bar" }
61 
+      agent.should_not be_valid
62 
+
63 
+      agent.options['recipients'] = "wut"
64 
+      agent.should_not be_valid
65 
+    end
66 
+  end
67 
+
68 
+  describe "#recipients" do
69 
+    it "defaults to the user's email address" do
70 
+      agent.recipients.should == [users(:jane).email]
71 
+    end
72 
+
73 
+    it "wraps a string with an array" do
74 
+      agent.options['recipients'] = 'bob@bob.com'
75 
+      agent.recipients.should == ['bob@bob.com']
76 
+    end
77 
+
78 
+    it "handles an array" do
79 
+      agent.options['recipients'] = ['bob@bob.com', 'jane@jane.com']
80 
+      agent.recipients.should == ['bob@bob.com', 'jane@jane.com']
81 
+    end
82 
+
83 
+    it "interpolates" do
84 
+      agent.options['recipients'] = "{{ username }}@{{ domain }}"
85 
+      agent.recipients('username' => 'bob', 'domain' => 'example.com').should == ["bob@example.com"]
86 
+    end
87 
+  end
88 
+end

+ 5 - 5
spec/support/shared_examples/liquid_interpolatable.rb
Просмотреть файл 

@@ -20,7 +20,7 @@ shared_examples_for LiquidInterpolatable do
20 20
21 21 
   describe "interpolating liquid templates" do
22 22 
     it "should work" do
23 
-      @checker.interpolate_options(@checker.options, @event.payload).should == {
23 
+      @checker.interpolate_options(@checker.options, @event).should == {
24 24 
           "normal" => "just some normal text",
25 25 
           "variable" => "hello",
26 26 
           "text" => "Some test with an embedded hello",
 
@@ -30,7 +30,7 @@ shared_examples_for LiquidInterpolatable do
30 30
31 31 
     it "should work with arrays", focus: true do
32 32 
       @checker.options = {"value" => ["{{variable}}", "Much array", "Hey, {{hello_world}}"]}
33 
-      @checker.interpolate_options(@checker.options, @event.payload).should == {
33 
+      @checker.interpolate_options(@checker.options, @event).should == {
34 34 
         "value" => ["hello", "Much array", "Hey, Hello world"]
35 35 
       }
36 36 
     end
 
@@ -38,7 +38,7 @@ shared_examples_for LiquidInterpolatable do
38 38 
     it "should work recursively" do
39 39 
       @checker.options['hash'] = {'recursive' => "{{variable}}"}
40 40 
       @checker.options['indifferent_hash'] = ActiveSupport::HashWithIndifferentAccess.new({'recursive' => "{{variable}}"})
41 
-      @checker.interpolate_options(@checker.options, @event.payload).should == {
41 
+      @checker.interpolate_options(@checker.options, @event).should == {
42 42 
           "normal" => "just some normal text",
43 43 
           "variable" => "hello",
44 44 
           "text" => "Some test with an embedded hello",
 
@@ -49,8 +49,8 @@ shared_examples_for LiquidInterpolatable do
49 49 
     end
50 50
51 51 
     it "should work for strings" do
52 
-      @checker.interpolate_string("{{variable}}", @event.payload).should == "hello"
53 
-      @checker.interpolate_string("{{variable}} you", @event.payload).should == "hello you"
52 
+      @checker.interpolate_string("{{variable}}", @event).should == "hello"
53 
+      @checker.interpolate_string("{{variable}} you", @event).should == "hello you"
54 54 
     end
55 55 
   end
56 56

+ 66 - 0
spec/support/shared_examples/web_request_concern.rb
Просмотреть файл 

@@ -0,0 +1,66 @@
1 
+require 'spec_helper'
2 
+
3 
+shared_examples_for WebRequestConcern do
4 
+  let(:agent) do
5 
+    _agent = described_class.new(:name => "some agent", :options => @valid_options || {})
6 
+    _agent.user = users(:jane)
7 
+    _agent
8 
+  end
9 
+
10 
+  describe "validations" do
11 
+    it "should be valid" do
12 
+      agent.should be_valid
13 
+    end
14 
+
15 
+    it "should validate user_agent" do
16 
+      agent.options['user_agent'] = nil
17 
+      agent.should be_valid
18 
+
19 
+      agent.options['user_agent'] = ""
20 
+      agent.should be_valid
21 
+
22 
+      agent.options['user_agent'] = "foo"
23 
+      agent.should be_valid
24 
+
25 
+      agent.options['user_agent'] = ["foo"]
26 
+      agent.should_not be_valid
27 
+
28 
+      agent.options['user_agent'] = 1
29 
+      agent.should_not be_valid
30 
+    end
31 
+
32 
+    it "should validate headers" do
33 
+      agent.options['headers'] = "blah"
34 
+      agent.should_not be_valid
35 
+
36 
+      agent.options['headers'] = ""
37 
+      agent.should be_valid
38 
+
39 
+      agent.options['headers'] = {}
40 
+      agent.should be_valid
41 
+
42 
+      agent.options['headers'] = { 'foo' => 'bar' }
43 
+      agent.should be_valid
44 
+    end
45 
+
46 
+    it "should validate basic_auth" do
47 
+      agent.options['basic_auth'] = "foo:bar"
48 
+      agent.should be_valid
49 
+
50 
+      agent.options['basic_auth'] = ["foo", "bar"]
51 
+      agent.should be_valid
52 
+
53 
+      agent.options['basic_auth'] = ""
54 
+      agent.should be_valid
55 
+
56 
+      agent.options['basic_auth'] = nil
57 
+      agent.should be_valid
58 
+
59 
+      agent.options['basic_auth'] = "blah"
60 
+      agent.should_not be_valid
61 
+
62 
+      agent.options['basic_auth'] = ["blah"]
63 
+      agent.should_not be_valid
64 
+    end
65 
+  end
66 
+end

+ 9 - 0
spec/support/vcr_support.rb
Просмотреть файл 

@@ -0,0 +1,9 @@
1 
+require 'vcr'
2 
+
3 
+VCR.configure do |c|
4 
+  c.cassette_library_dir = 'spec/cassettes'
5 
+  c.allow_http_connections_when_no_cassette = true
6 
+  c.hook_into :webmock
7 
+  c.default_cassette_options = { record: :new_episodes}
8 
+  c.configure_rspec_metadata!
9 
+end

Implement FtpsiteAgent. · e4dc6a31c3 - Gogs J1X
Главная Обзор Помощь
Зарегистрироваться Войти
J1X
/
huginn
Следить 2
В избранное 0
Ответвить 0

Просмотр исходного кода
Implement FtpsiteAgent.

This agent checks a FTP site and creates Events based on newly uploaded
files in a directory.

Akinori MUSHA лет %!s(int64=12): %!d(string=назад)
Родитель
ff546b6671
Сommit
e4dc6a31c3
2 измененных файлов с 293 добавлено и 0 удалено
Показать статистику Diff
  1. 214 0
      app/models/agents/ftpsite_agent.rb
  2. 79 0
      spec/models/agents/ftpsite_agent_spec.rb

+ 214 - 0
app/models/agents/ftpsite_agent.rb
Просмотреть файл 

@@ -0,0 +1,214 @@
1 
+require 'net/ftp'
2 
+require 'uri'
3 
+require 'time'
4 
+
5 
+module Agents
6 
+  class FtpsiteAgent < Agent
7 
+    cannot_receive_events!
8 
+
9 
+    default_schedule "every_12h"
10 
+
11 
+    description <<-MD
12 
+      The FtpsiteAgent checks a FTP site and creates Events based on newly uploaded files in a directory.
13 
+
14 
+      Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names.
15 
+
16 
+      Login credentials can be included in `url` if authentication is required.
17 
+
18 
+      Only files with a last modification time later than the `after` value, if specifed, are notified.
19 
+    MD
20 
+
21 
+    event_description <<-MD
22 
+      Events look like this:
23 
+
24 
+          {
25 
+            "url": "ftp://example.org/pub/releases/foo-1.2.tar.gz",
26 
+            "filename": "foo-1.2.tar.gz",
27 
+            "timestamp": "2014-04-10T22:50:00Z"
28 
+          }
29 
+    MD
30 
+
31 
+    def working?
32 
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
33 
+    end
34 
+
35 
+    def default_options
36 
+      {
37 
+          'expected_update_period_in_days' => "1",
38 
+          'url' => "ftp://example.org/pub/releases/",
39 
+          'patterns' => [
40 
+            'foo-*.tar.gz',
41 
+          ],
42 
+          'after' => Time.now.iso8601,
43 
+      }
44 
+    end
45 
+
46 
+    def validate_options
47 
+      # Check for required fields
48 
+      begin
49 
+        url = options['url']
50 
+        String === url or raise
51 
+        uri = URI(url)
52 
+        URI::FTP === uri or raise
53 
+        errors.add(:base, "url must end with a slash") unless uri.path.end_with?('/')
54 
+      rescue
55 
+        errors.add(:base, "url must be a valid FTP URL")
56 
+      end
57 
+
58 
+      patterns = options['patterns']
59 
+      case patterns
60 
+      when Array
61 
+        if patterns.empty?
62 
+          errors.add(:base, "patterns must not be empty")
63 
+        end
64 
+      when nil, ''
65 
+        errors.add(:base, "patterns must be specified")
66 
+      else
67 
+        errors.add(:base, "patterns must be an array")
68 
+      end
69 
+
70 
+      # Check for optional fields
71 
+      if (timestamp = options['timestamp']).present?
72 
+        begin
73 
+          Time.parse(timestamp)
74 
+        rescue
75 
+          errors.add(:base, "timestamp cannot be parsed as time")
76 
+        end
77 
+      end
78 
+
79 
+      if options['expected_update_period_in_days'].present?
80 
+        errors.add(:base, "Invalid expected_update_period_in_days format") unless is_positive_integer?(options['expected_update_period_in_days'])
81 
+      end
82 
+    end
83 
+
84 
+    def check
85 
+      transaction do |found|
86 
+        each_entry { |filename, mtime|
87 
+          found[filename, mtime]
88 
+        }
89 
+      end
90 
+    end
91 
+
92 
+    def each_entry
93 
+      patterns = options['patterns']
94 
+
95 
+      after =
96 
+        if str = options['after']
97 
+          Time.parse(str)
98 
+        else
99 
+          Time.at(0)
100 
+        end
101 
+
102 
+      open_ftp(base_uri) do |ftp|
103 
+        log "Listing the directory"
104 
+        # Do not use a block style call because we need to call other
105 
+        # commands during iteration.
106 
+        list = ftp.list('-a')
107 
+
108 
+        month2year = {}
109 
+
110 
+        list.each do |line|
111 
+          mon, day, smtn, rest = line.split(' ', 9)[5..-1]
112 
+
113 
+          # Remove symlink target part if any
114 
+          filename = rest[/\A(.+?)(?:\s+->\s|\z)/, 1]
115 
+
116 
+          patterns.any? { |pattern|
117 
+            File.fnmatch?(pattern, filename)
118 
+          } or next
119 
+
120 
+          case smtn
121 
+          when /:/
122 
+            if year = month2year[mon]
123 
+              mtime = Time.parse("#{mon} #{day} #{year} #{smtn} GMT")
124 
+            else
125 
+              log "Getting mtime of #{filename}"
126 
+              mtime = ftp.mtime(filename)
127 
+              month2year[mon] = mtime.year
128 
+            end
129 
+          else
130 
+            # Do not bother calling MDTM for old files.  Losing the
131 
+            # time part only makes a timestamp go backwards, meaning
132 
+            # that it will trigger no new event.
133 
+            mtime = Time.parse("#{mon} #{day} #{smtn} GMT")
134 
+          end
135 
+
136 
+          after < mtime or next
137 
+
138 
+          yield filename, mtime
139 
+        end
140 
+      end
141 
+    end
142 
+
143 
+    def open_ftp(uri)
144 
+      ftp = Net::FTP.new
145 
+
146 
+      log "Connecting to #{uri.host}#{':%d' % uri.port if uri.port != uri.default_port}"
147 
+      ftp.connect(uri.host, uri.port)
148 
+
149 
+      user =
150 
+        if str = uri.user
151 
+          URI.decode(str)
152 
+        else
153 
+          'anonymous'
154 
+        end
155 
+      password =
156 
+        if str = uri.password
157 
+          URI.decode(str)
158 
+        else
159 
+          'anonymous@'
160 
+        end
161 
+      log "Logging in as #{user}"
162 
+      ftp.login(user, password)
163 
+
164 
+      ftp.passive = true
165 
+
166 
+      path = uri.path.chomp('/')
167 
+      log "Changing directory to #{path}"
168 
+      ftp.chdir(path)
169 
+
170 
+      yield ftp
171 
+    ensure
172 
+      log "Closing the connection"
173 
+      ftp.close
174 
+    end
175 
+
176 
+    def base_uri
177 
+      @base_uri ||= URI(options['url'])
178 
+    end
179 
+
180 
+    def transaction
181 
+      known_entries = memory['known_entries'] || {}
182 
+      found_entries = {}
183 
+      new_files = []
184 
+
185 
+      yield proc { |filename, mtime|
186 
+        found_entries[filename] = misotime = mtime.utc.iso8601
187 
+        unless prev = known_entries[filename] and misotime <= prev
188 
+          new_files << filename
189 
+        end
190 
+      }
191 
+
192 
+      new_files.sort_by { |filename|
193 
+        found_entries[filename]
194 
+      }.each { |filename|
195 
+        create_event :payload => {
196 
+          'url' => (base_uri + filename).to_s,
197 
+          'filename' => filename,
198 
+          'timestamp' => found_entries[filename],
199 
+        }
200 
+      }
201 
+
202 
+      memory['known_entries'] = found_entries
203 
+      save!
204 
+    end
205 
+
206 
+    private
207 
+
208 
+    def is_positive_integer?(value)
209 
+      Integer(value) >= 0
210 
+    rescue
211 
+      false
212 
+    end
213 
+  end
214 
+end

+ 79 - 0
spec/models/agents/ftpsite_agent_spec.rb
Просмотреть файл 

@@ -0,0 +1,79 @@
1 
+require 'spec_helper'
2 
+require 'time'
3 
+
4 
+describe Agents::FtpsiteAgent do
5 
+  describe "checking anonymous FTP" do
6 
+    before do
7 
+      @site = {
8 
+        'expected_update_period_in_days' => 1,
9 
+        'url' => "ftp://ftp.example.org/pub/releases/",
10 
+        'patterns' => ["example-*.tar.gz"],
11 
+      }
12 
+      @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2)
13 
+      @checker.user = users(:bob)
14 
+      @checker.save!
15 
+      stub(@checker).each_entry.returns { |block|
16 
+        block.call("example-latest.tar.gz", Time.parse("2014-04-01T10:00:01Z"))
17 
+        block.call("example-1.0.tar.gz",    Time.parse("2013-10-01T10:00:00Z"))
18 
+        block.call("example-1.1.tar.gz",    Time.parse("2014-04-01T10:00:00Z"))
19 
+      }
20 
+    end
21 
+
22 
+    describe "#check" do
23 
+      it "should validate the integer fields" do
24 
+        @checker.options['expected_update_period_in_days'] = "nonsense"
25 
+        lambda { @checker.save! }.should raise_error;
26 
+        @checker.options = @site
27 
+      end
28 
+
29 
+      it "should check for changes and save known entries in memory" do
30 
+        lambda { @checker.check }.should change { Event.count }.by(3)
31 
+        @checker.memory['known_entries'].tap { |known_entries|
32 
+          known_entries.size.should == 3
33 
+          known_entries.sort_by(&:last).should == [
34 
+            ["example-1.0.tar.gz",    "2013-10-01T10:00:00Z"],
35 
+            ["example-1.1.tar.gz",    "2014-04-01T10:00:00Z"],
36 
+            ["example-latest.tar.gz", "2014-04-01T10:00:01Z"],
37 
+          ]
38 
+        }
39 
+
40 
+        Event.last(2).first.payload.should == {
41 
+          'url' => 'ftp://ftp.example.org/pub/releases/example-1.1.tar.gz',
42 
+          'filename' => 'example-1.1.tar.gz',
43 
+          'timestamp' => '2014-04-01T10:00:00Z',
44 
+        }
45 
+
46 
+        lambda { @checker.check }.should_not change { Event.count }
47 
+
48 
+        stub(@checker).each_entry.returns { |block|
49 
+          block.call("example-latest.tar.gz", Time.parse("2014-04-02T10:00:01Z"))
50 
+
51 
+          # In the long list format the timestamp may look going
52 
+          # backwards after six months: Oct 01 10:00 -> Oct 01 2013
53 
+          block.call("example-1.0.tar.gz",    Time.parse("2013-10-01T00:00:00Z"))
54 
+
55 
+          block.call("example-1.1.tar.gz",    Time.parse("2014-04-01T10:00:00Z"))
56 
+          block.call("example-1.2.tar.gz",    Time.parse("2014-04-02T10:00:00Z"))
57 
+        }
58 
+        lambda { @checker.check }.should change { Event.count }.by(2)
59 
+        @checker.memory['known_entries'].tap { |known_entries|
60 
+          known_entries.size.should == 4
61 
+          known_entries.sort_by(&:last).should == [
62 
+            ["example-1.0.tar.gz",    "2013-10-01T00:00:00Z"],
63 
+            ["example-1.1.tar.gz",    "2014-04-01T10:00:00Z"],
64 
+            ["example-1.2.tar.gz",    "2014-04-02T10:00:00Z"],
65 
+            ["example-latest.tar.gz", "2014-04-02T10:00:01Z"],
66 
+          ]
67 
+        }
68 
+
69 
+        Event.last(2).first.payload.should == {
70 
+          'url' => 'ftp://ftp.example.org/pub/releases/example-1.2.tar.gz',
71 
+          'filename' => 'example-1.2.tar.gz',
72 
+          'timestamp' => '2014-04-02T10:00:00Z',
73 
+        }
74 
+
75 
+        lambda { @checker.check }.should_not change { Event.count }
76 
+      end
77 
+    end
78 
+  end
79 
+end